package com.duowan.mspider.processor;

import com.duowan.mspider.download.MySeleniumDownloader;
import com.duowan.mspider.util.Constant;
import com.duowan.mspider.util.Constant.KeyWordsFile;
import com.duowan.mspider.util.Constant.SearchEngine;
import com.duowan.mspider.util.FileUtils;
import org.apache.commons.lang3.StringUtils;
import us.codecraft.webmagic.Page;
import us.codecraft.webmagic.Site;
import us.codecraft.webmagic.Spider;
import us.codecraft.webmagic.pipeline.ConsolePipeline;
import us.codecraft.webmagic.processor.PageProcessor;

import java.io.File;
import java.net.URLDecoder;
import java.util.List;

/**
 * 神马
 */
public class MSmRepoPageProcessor implements PageProcessor {
	private Site site = Site.me().setRetryTimes(1).setSleepTime(300).setCharset("utf-8")
								.setUserAgent("Mozilla/5.0 (Linux; U; Android 4.0.4; en-gb; GT-I9300 Build/IMM76D) AppleWebKit/534.30 (KHTML, like Gecko) Version/4.0 Mobile Safari/534.30");

	private Constant.SearchEngine engine;
	private KeyWordsFile keyWordsFile;

	public MSmRepoPageProcessor(SearchEngine engine, KeyWordsFile keyWordsFile) {
		this.engine = engine;
		this.keyWordsFile = keyWordsFile;
	}

	public SearchEngine getEngine() {
		return engine;
	}

	public KeyWordsFile getKeyWordsFile() {
		return keyWordsFile;
	}

	@Override
	public void process(Page page) {
		List<String> listUrls = page.getHtml().xpath(engine.xpath).all();
		for (String url : listUrls) {
			try {
				if (StringUtils.isBlank(url) || url.startsWith("https")) continue;
				File file = new File(keyWordsFile.getSmTempFile());

				String adLink = URLDecoder.decode(url, "UTF-8");
				System.out.println(adLink);
				FileUtils.write(file, adLink);
			} catch (Exception e) {
				e.printStackTrace();
			}
		}


	}

	@Override
	public Site getSite() {
		return site;
	}

	public static void main(String[] args) throws Exception {
		System.setProperty("selenuim_config", "config.ini");

		for (KeyWordsFile keyWordsFile : KeyWordsFile.values()) {
			File tempFile = new File(keyWordsFile.getSmTempFile());
			if (tempFile.exists()) tempFile.delete();

			File fileName = new File(keyWordsFile.srcFileName);
			List<String> keywords = FileUtils.read(fileName);

			MSmRepoPageProcessor processor = new MSmRepoPageProcessor(SearchEngine.Sm, keyWordsFile);
			Spider spider = Spider.create(processor);
			for (String keyword : keywords) {
				spider.addUrl(processor.getEngine().url + keyword);
			}
			//浏览器驱动（动态网页信息通过模拟浏览器启动获取）
			MySeleniumDownloader downloader = new MySeleniumDownloader("chromedriver.exe");
			spider.setDownloader(downloader).addPipeline(new ConsolePipeline()).thread(1).run();
		}

	}
}